package com.foo.DataPreprocessing;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.LowerCaseTokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.PorterStemFilter;
import java.io.StringReader;
import java.io.IOException;
import java.util.ArrayList;

public class LuceneCleanUp 
{
	private Analyzer stand_analyzer = new StandardAnalyzer();
	private TokenStream stream;
    private PorterStemFilter stemmer;
	
	/*
	 * call this function to tokenize, remove stop words, white spaces, stemming on the string.
	 */
	@SuppressWarnings("deprecation")
	public ArrayList<String> analyze(String text) throws IOException {
		
		ArrayList<String> tokenList = new ArrayList<String>();
		
        System.out.println("Analzying " + text);
        System.out.print("\t\t");
        stream = stand_analyzer.tokenStream("contents", new StringReader(text));
        stemmer = new PorterStemFilter(stream);
        
        while (true) 
        {
        Token token = stemmer.next();
        if (token == null) break;
        tokenList.add(token.termText());
        System.out.print("[" + token.termText() + "] ");
        }
        
        System.out.println("\n");
        return tokenList;
    }
	
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		LuceneCleanUp lucene = new LuceneCleanUp();
		ArrayList<String> wordList = null;
		try {
			wordList = lucene.analyze("The US trade deficit has exploding 19% to a record $55.8bn as oil costs drove imports higher, according to a latest figures");
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		for(int j=0;j<wordList.size();j++) 
        {
	        System.out.println("Preprocessed Data is:" +wordList.get(j));
		}
	}
}
